package com.putao.spider;

import java.util.List;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.pipeline.ConsolePipeline;
import us.codecraft.webmagic.pipeline.JsonFilePipeline;
import us.codecraft.webmagic.processor.PageProcessor;

public class ZhihuSpider implements PageProcessor {

	private Site site = Site.me().setRetryTimes(3).setSleepTime(5000);
	Integer count = 0;

	@Override
	public void process(Page page) {
		//分类链接
		List<String> catLinks = page.getHtml().xpath("//ul[@class=\"zm-topic-cat-main clearfix\"]/").links()
				.regex("https://www\\.zhihu\\.com/topics#\\W+").all();
		//话题链接
		List<String> topicLinks = page.getHtml().xpath("//div[@class=\"zm-topic-cat-sub\"]/").links()
				.regex("https://www\\.zhihu\\.com/topic/\\d+").all();
		//问题链接
		List<String> questionLinks = page.getHtml().xpath("//div[@class=\"zm-topic-list-container\"]/").links()
				.regex("https://www\\.zhihu\\.com/question/\\d+").all();
		page.addTargetRequests(catLinks);
		page.addTargetRequests(topicLinks);
		page.addTargetRequests(questionLinks);
		String temp = page.getHtml()
				.xpath("//div[@class=\"zh-question-followers-sidebar\"]/div[@class=\"zg-gray-normal\"]/a/strong/text()")
				.toString();
		if (temp != null && new Integer(temp) > 1000) {
			page.putField("focusCount", temp);
		}
	}

	@Override
	public Site getSite() {
		site.setCharset("UTF-8");
		site.addCookie("z_c0",
				"Mi4wQUFDQVA0VS1BQUFBVUFCUjBMUFVDaGNBQUFCaEFsVk5TUW1rV0FEeUoyVGpqT29hVFlqeVU4TEVKc3h4SHRjSW1n|1484553292|5c837fd8ca61349a8788136ddd445a69c3cc5552");
		return site;
	}

	public static void main(String[] args) {
		Spider.create(new ZhihuSpider()).addUrl("https://www.zhihu.com/topics")
				.addPipeline(new JsonFilePipeline("D:\\webmagic\\")).addPipeline(new ConsolePipeline()).thread(5).run();
	}
}